package com.zx.ziroom;

import java.io.Closeable;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @Project: my_spider
 * @Title: Main
 * @Description: 抓取自如全部的房源
 * @author: zhangxue
 * @date: 2017年12月4日下午9:46:40
 * @company: webyun
 * @Copyright: Copyright (c) 2015
 * @version v1.0
 */
public class Main {
	
	private static Logger LOGGER = LoggerFactory.getLogger(Main.class);
	
	public static void main(String[] args) {
		RoomDao roomDao = new RoomDao();
		try {
			roomDao.createTable();
		} catch (SQLException e) {
			LOGGER.error("自如表初始化异常");
			LOGGER.error(e.getMessage(), e);
		}
		
		// 1. 遍历所有的城市
		ZiRoomCity[] values = ZiRoomCity.values();
		
		int cityIndex = 0;//城市的序号
		int pageIndex = 0;//该城市第几页的序号
		
		for (ZiRoomCity ziRoomCity : values) {
			String cityName = ziRoomCity.getName();
			String cityUrl = ziRoomCity.getUrl();
			pageIndex = 0;//初始化该城市第几页的序号
			
			LOGGER.info("第一步:开始获取第【{}】个城市【{}】的数据，连接是【{}】", cityIndex + 1, cityName, cityUrl);
			// 2. 获得这个城市所有的分页的URL-有几个分页
			try {
				boolean isNextPage = true;//是否解析下一页
				// 注意：自如在官网上页数最多显示50页，但是其实分数会大于这个数字，所以对比上一页和本页产品的URL,如果完全相同，这说明这个城市已经全部结束了
				List<String> roomUrlPrev = new ArrayList<>();//上一页的URL
				List<String> roomUrlPage = new ArrayList<>();//本页的URL
				
				while (isNextPage) {
					String pageUrl = cityUrl + "?p=" + (pageIndex + 1);
					//获取这一页的房子的url
					try {
						roomUrlPage = getRoomUrls(pageUrl);
						for (String roomUrl : roomUrlPage) {
							
							try {
								Room roomModel = getRoomModel(roomUrl, cityName);
								roomDao.insert(roomModel);
								LOGGER.info("开始获取【{}】第【{}】页的数据，url={}", cityName, pageIndex + 1, roomUrl);
							} catch (Exception e) {
								e.printStackTrace();
							}
						}
						isNextPage = isNextPage(roomUrlPrev, roomUrlPage);
					} catch (Exception e) {
						LOGGER.error("第三步:获取room url异常，位置cityIndex={}, pageIndex={}", cityIndex, pageIndex);
						LOGGER.error(e.getMessage(), e);
					}
					pageIndex ++;
					roomUrlPrev = roomUrlPage;//将这一页的url，赋值为上一个的url
					
					if(pageIndex > 200)//页数不可能太大的，超过200就是异常了
						isNextPage = false;
				}
				
			} catch (Exception e) {
				LOGGER.error("第二步:获取这个城市所有分页异常，位置cityIndex={}, pageIndex={}", cityIndex, pageIndex);
				LOGGER.error(e.getMessage(), e);
			}
			cityIndex ++;
		}
	}
	
	
	public static List<String> getRoomUrls(String pageUrl) throws Exception {
		CloseableHttpClient httpclient =  null;
		CloseableHttpResponse response = null;
		
		try {
			httpclient =  HttpClients.createDefault();// 创建httpclient实例
			HttpGet httpget = new HttpGet(pageUrl); // 创建httpget实例
		    response = httpclient.execute(httpget); // 执行get请求
			
	        HttpEntity entity=response.getEntity(); // 获取返回实体
	        String content=EntityUtils.toString(entity, "utf-8");
	        
	        Document doc=Jsoup.parse(content); // 解析网页 得到文档对象
	        
	        Elements linkElements=doc.select("#houseList li .txt h3 a"); //通过选择器查找所有博客链接DOM
	        
	        List<String> roomUrls = new ArrayList<>(linkElements.size());
	        
	        for(Element e:linkElements){
	        	roomUrls.add(e.attr("href").replace("//", "http://"));
	        }
	        return roomUrls;
		} catch (Exception e){
			throw e;
		} finally {
			close(response);
			close(httpclient);
		}
	}
	
	public static Room getRoomModel(String roomUrl, String cityName) throws Exception {
		CloseableHttpClient httpclient =  null;
		CloseableHttpResponse response = null;
		
		try {
			httpclient =  HttpClients.createDefault();// 创建httpclient实例
			HttpGet httpget = new HttpGet(roomUrl); // 创建httpget实例
		    response = httpclient.execute(httpget); // 执行get请求
			
	        HttpEntity entity=response.getEntity(); // 获取返回实体
	        String content=EntityUtils.toString(entity, "utf-8");
	        
	        Document doc=Jsoup.parse(content); // 解析网页 得到文档对象
	        
	        
	        String roomName = doc.getElementsByClass("room_name").get(0).getElementsByTag("h2").text();
	        String roomPrice = doc.getElementById("room_price").text().replace("￥", "");
	        
	        // 获得详情页
	        Elements lis = doc.getElementsByClass("detail_room").get(0).getElementsByTag("li");
	        
	        String area = lis.get(0).text().substring(3).trim().split(" ")[0];
	        
	        String orientation = lis.get(1).text().substring(3).trim();
	        
	        String roomType = lis.get(2).text().substring(3).split(" ")[1];
	        
	    	String priceType = lis.get(2).getElementsByTag("span").get(0).text();
	    	
	    	String floor = lis.get(3).text().substring(3).trim();
	        
	        return new Room(null, roomName, roomPrice, cityName, roomUrl, area, orientation, roomType, priceType, floor);
		} catch (Exception e){
			throw e;
		} finally {
			close(response);
			close(httpclient);
		}
	}
	/**
	 * @Title: 关闭资源
	 * @param closeable
	 * @date: 2017年12月3日下午9:13:07
	 */
	public static void close(Closeable closeable){
		if(closeable != null)
			try {
				closeable.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
	} 
	
	/**
	 * @Title: 是否继续解析下一页
	 * @param prevUrl
	 * @param pageUrl
	 * @Description: TODO
	 * @date: 2017年12月3日下午9:39:56
	 */
	public static boolean isNextPage (List<String> prevUrl, List<String> pageUrl) {
		return !prevUrl.containsAll(pageUrl);
	}
	
}
